{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'\\n本脚本使用gensim来读取中文文学词向量sgns.literature.word\\n\\n'"
      ]
     },
     "metadata": {},
     "execution_count": 1
    }
   ],
   "source": [
    "\"\"\"\n",
    "本脚本使用gensim来读取中文文学词向量sgns.literature.word\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from gensim.models import KeyedVectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cn_word_path=r'E:\\PycharmOut\\CGAI\\Chinese_Word_Vectors\\sgns.literature.word'\n",
    "cn_model=KeyedVectors.load_word2vec_format(cn_word_path,binary=False,unicode_errors='ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "187959"
      ]
     },
     "metadata": {},
     "execution_count": 5
    }
   ],
   "source": [
    "len(cn_model.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_list=['距离',\n",
    " '川沙',\n",
    " '公路',\n",
    " '较近',\n",
    " '公交',\n",
    " '指示',\n",
    " '蔡陆线',\n",
    " '会',\n",
    " '非常',\n",
    " '麻烦',\n",
    " '建议',\n",
    " '路线',\n",
    " '房间',\n",
    " '较为简单']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "1414\n103431\n3472\n较近 0\n40039\n3549\n蔡陆线 0\n64\n271\n1874\n1922\n4333\n636\n182979\n"
     ]
    }
   ],
   "source": [
    "#第一次使用载入后的词向量模型cn_model时，会加载到内存里会比较慢，大概10分钟（机器不好）。内存一下上了10多个G。但是加载完成后，获取词的向量或者索引就很快了。\n",
    "#通过cn_model.vocab['词'].index可以获取该词在词向量空间中的索引。\n",
    "\n",
    "for i in test_list:\n",
    "    try:\n",
    "        print(cn_model.vocab[i].index)\n",
    "    except Exception as no_word_ERR:\n",
    "        print(i,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<gensim.models.keyedvectors.Vocab at 0x2796db6e978>"
      ]
     },
     "metadata": {},
     "execution_count": 10
    }
   ],
   "source": [
    "cn_model.vocab['公路']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(300,)"
      ]
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "source": [
    "#直接传入词，就会得到词向量。此时已经获取到你所想要的词向量了\n",
    "test_pre=cn_model['公路']\n",
    "test_pre.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'，'"
      ]
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "source": [
    "#通过index方向获取对应的中文词\n",
    "i0=cn_model.index2word[0]\n",
    "i0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'的'"
      ]
     },
     "metadata": {},
     "execution_count": 14
    }
   ],
   "source": [
    "i1=cn_model.index2word[1]\n",
    "i1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(7000, 3000)"
      ]
     },
     "metadata": {},
     "execution_count": 17
    }
   ],
   "source": [
    "#开始\n",
    "#读取所有文本放入到的x_train\n",
    "cd=os.path.abspath('')\n",
    "pos_file_path=os.path.join(cd,'pinglun','pos.txt')\n",
    "neg_file_path=os.path.join(cd,'pinglun','neg.txt')\n",
    "\n",
    "x_train=[]\n",
    "\n",
    "pos_list=[]\n",
    "with open(pos_file_path,'r',encoding='utf8') as r1:\n",
    "    pos_list=r1.readlines()\n",
    "\n",
    "neg_list=[]\n",
    "with open(neg_file_path,'r',encoding='utf8') as r2:\n",
    "    neg_list=r2.readlines()\n",
    "\n",
    "len(pos_list),len(neg_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'1    房间设施难以够得上五星级，服务还不错，有送水果。\\n'"
      ]
     },
     "metadata": {},
     "execution_count": 18
    }
   ],
   "source": [
    "pos_list[6999]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "3000"
      ]
     },
     "metadata": {},
     "execution_count": 19
    }
   ],
   "source": [
    "#正样本有7000个，截取到3000个\n",
    "pos_list=pos_list[0:3000]\n",
    "len(pos_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "6000"
      ]
     },
     "metadata": {},
     "execution_count": 21
    }
   ],
   "source": [
    "#将正负样本拼接在一起，一起操作\n",
    "x_train.extend(pos_list)\n",
    "x_train.extend(neg_list)\n",
    "len(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#将样本进行去标点，去停顿词，去空格,去换行操作\n",
    "import jieba\n",
    "import jieba.analyse as ja\n",
    "import re\n",
    "\n",
    "\n",
    "def drop_punctuation(text):\n",
    "    punc = '~`!#$%^&*()_+-=|\\';\":/.,?><~·！@#￥%……&*（）——+-=“：’；、。，？》《{} \\n'\n",
    "    new_text=re.sub(r\"[%s]+\" %punc, \"\",text)\n",
    "    return new_text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'距离川沙公路较近但是公交指示不对如果是蔡陆线的话会非常麻烦建议用别的路线房间较为简单'"
      ]
     },
     "metadata": {},
     "execution_count": 24
    }
   ],
   "source": [
    "x_train=[ drop_punctuation(i) for i in x_train]\n",
    "x_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}